﻿//  ************************************************************************** 
//  *                                                                        * 
//  *     u S C A N N E R  General purpose tokenizer for .NET                * 
//  *                                                                        *
//  *  Usage:                                                                *
//  *    TScanner sc = new TScanner();                                       *
//  *    sc.stream = <A stream>; // eg sc.stream = File.OpenRead (fileName); *
//  *    sc.startScanner();                                                  * 
//  *    sc.nextToken();                                                     * 
//  *    if (sc.token == TScanner.codes.tEndOFStreamToken)             *
//  *       exit;                                                            *
//  *                                                                        * 
//  *  Copyright (c) 2004 by Herbert Sauro                                   *
//  *  Licenced under the Artistic open source licence                       *
//  *                                                                        * 
//  ************************************************************************** 

using System;
using System.IO;
using System.Collections;

namespace Athena.Functions
{

    // Declare a Scanner exception type
    public class EScannerException : System.ApplicationException
    {
        public EScannerException(string message)
            : base(message)
        {
        }
    }

    // Different types of recognised tokens
    public enum code
    {
        tEmptyToken, tEndOfStreamToken, tIntToken, tDoubleToken,
        tComplexToken, tStringToken, tWordToken, tEolToken, tSemiColonToken,
        tCommaToken, tEqualsToken, tPlusToken, tMinusToken, tMultToken, tDivToken,
        tLParenToken, tRParenToken, tLBracToken, tRBracToken, tLCBracToken, tRCBracToken,
        tOrToken, tAndToken, tNotToken, tXorToken, tTimeWord1, tTimeWord2, tTimeWord3, tColonToken,
        tPowerToken, tLessThanToken, tLessThanOrEqualToken, tMoreThanToken,
        tMoreThanOrEqualToken, tNotEqualToken, tReversibleArrow, tIrreversibleArrow,
        tStartComment,
        tInternalToken, tVarToken, tVolToken,
        tExternalToken, tExtToken,
        tParameterToken, tIfToken,
        tDollarToken,
        tUnaryMinusToken,
        tPrintToken,
        tWhileToken, tdefnToken, tEndToken, tTimeStartToken, tTimeEndToken, tNumPointsToken,
        tSimulateToken, tPointToken
    }


    // This is currently used to store the previous token and support simple look ahead
    public class TToken
    {
        public code tokenCode;
        public string tokenString;
        public int tokenInteger;
        public double tokenDouble;
        public double tokenValue;  // Used to retrieve int or double
    }


    // -------------------------------------------------------------------
    //    Start of TScanner Class
    // -------------------------------------------------------------------

    public class TScanner
    {
        const char EOFCHAR = '\x7F';  // Deemed end of string marker, used internally
        const char CR = (char)13;
        const char LF = (char)10;

        enum TCharCode
        {
            cLETTER,
            cDIGIT,
            cPOINT,
            cDOUBLEQUOTE,
            cUNDERSCORE,
            cSPECIAL,
            cWHITESPACE,
            cETX
        }

        // private variables
        private TCharCode[] FCharTable = new TCharCode[255];
        private byte[] buffer = new byte[255];   // Input buffer
        private int bufferPtr;                   // Index of position in buffer containing current char
        private int bufferLength;

        private int yylineno;            // Current line number

        private Stream FStream;
        private code ftoken;
        private Hashtable wordTable;
        private Queue tokenQueue;

        public char fch;   // Current character read

        // Public Variables

        public TToken currentToken;
        public TToken previousToken;

        public bool IgnoreNewLines = true;
        public bool FromQueue = true;

        public string tokenString;
        public int tokenInteger;
        public double tokenDouble;
        public double tokenScalar;  // Used to retrieve int or double

        public static string timeWord1 = "time";
        public static string timeWord2 = "Time";
        public static string timeWord3 = "TIME";

        // Create a readonly property for the current line number
        public int lineNumber
        {
            get
            {
                return yylineno;
            }
        }

        // writeonly stream property 
        public Stream stream
        {
            set
            {
                FStream = value;
            }
            get
            {
                return FStream;
            }
        }

        // readonly current token property
        public code token
        {
            get
            {
                return ftoken;
            }
        }

        // -------------------------------------------------------------------
        //      Constructor
        // -------------------------------------------------------------------

        public TScanner()
        {
            wordTable = new Hashtable();
            previousToken = new TToken();
            currentToken = new TToken();
            tokenQueue = new Queue();
            initScanner();
        }

        private void initScanner()
        {
            char ch;
            for (ch = '\x00'; ch < '\xFF'; ch++)
                FCharTable[ch] = TCharCode.cSPECIAL;
            for (ch = '0'; ch <= '9'; ch++)
                FCharTable[ch] = TCharCode.cDIGIT;
            for (ch = 'A'; ch <= 'Z'; ch++)
                FCharTable[ch] = TCharCode.cLETTER;
            for (ch = 'a'; ch <= 'z'; ch++)
                FCharTable[ch] = TCharCode.cLETTER;

            FCharTable['.'] = TCharCode.cPOINT;
            FCharTable['"'] = TCharCode.cDOUBLEQUOTE;
            FCharTable['_'] = TCharCode.cUNDERSCORE;
            FCharTable['\t'] = TCharCode.cWHITESPACE;
            FCharTable[' '] = TCharCode.cWHITESPACE;
            FCharTable[EOFCHAR] = TCharCode.cETX;

            wordTable.Add("and", code.tAndToken);
            wordTable.Add("or", code.tOrToken);
            wordTable.Add("not", code.tNotToken);
            wordTable.Add("xor", code.tXorToken);

            wordTable.Add("defn", code.tdefnToken);
            wordTable.Add(timeWord1, code.tTimeWord1);
            wordTable.Add(timeWord2, code.tTimeWord2);
            wordTable.Add(timeWord3, code.tTimeWord3);
            wordTable.Add("if", code.tIfToken);
            wordTable.Add("while", code.tWhileToken);
            wordTable.Add("end", code.tEndToken);
            wordTable.Add("internal", code.tInternalToken);
            wordTable.Add("var", code.tVarToken);
            wordTable.Add("vol", code.tVolToken);
            wordTable.Add("external", code.tExternalToken);
            wordTable.Add("ext", code.tExtToken);
            wordTable.Add("parameter", code.tParameterToken);
            wordTable.Add("timeStart", code.tTimeStartToken);
            wordTable.Add("timeEnd", code.tTimeEndToken);
            wordTable.Add("numPoints", code.tNumPointsToken);
            wordTable.Add("simulate", code.tSimulateToken);
            wordTable.Add("print", code.tPrintToken);
        }

        // Must be called before using nextToken()
        public void startScanner()
        {
            yylineno = 1;
            bufferPtr = 0;
            nextChar();
        }

        private char getCharFromBuffer()
        {
            // If the buffer is empty, read a new chuck of text from the
            // input stream, this might be a stream or console
            if (bufferPtr == 0)
            {
                // Read a chunck of data from the input stream
                bufferLength = (char)FStream.Read(buffer, 0, 255);
                if (bufferLength == 0)
                    return EOFCHAR;
            }

            char ch = (char)buffer[bufferPtr];
            bufferPtr++;
            if (bufferPtr >= bufferLength)
                bufferPtr = 0;  // Indicates the buffer is empty
            return ch;
        }

        // -------------------------------------------------------------------
        // Fetches next character from input stream and filters NL if required 
        // -------------------------------------------------------------------

        public char nextChar()
        {
            fch = getCharFromBuffer();
            if (IgnoreNewLines)
            {
                // Turn any CFs or LFs into space characters
                if (fch == CR)
                {
                    yylineno++;
                    fch = ' ';
                    return fch;
                }

                if (fch == LF)
                    fch = ' ';
                return fch;
            }
            else
            {
                if (fch == CR)
                    yylineno++;
            }
            return fch;
        }


        // -------------------------------------------------------------------
        // Skips any blanks, ie TAB, ' '
        // -------------------------------------------------------------------

        public void skipBlanks()
        {
            while (FCharTable[fch] == TCharCode.cWHITESPACE)
            {
                if ((fch == LF) || (fch == CR))
                    return;
                nextChar();
            }
        }

        // -------------------------------------------------------------------
        // Scan for a word, words start with letter or underscore then continue
        // with letters, digits or underscore
        // -------------------------------------------------------------------   

        private void getWord()
        {
            while ((FCharTable[fch] == TCharCode.cLETTER)
                || (FCharTable[fch] == TCharCode.cDIGIT)
                || (FCharTable[fch] == TCharCode.cUNDERSCORE))
            {
                tokenString = tokenString + fch;  // Inefficient but convenient
                nextChar();
            }

            //if (wordTable.Contains(tokenString))
            //    ftoken = (code)wordTable[tokenString];
            //else
                ftoken = code.tWordToken;
        }

        // -------------------------------------------------------------------
        // Scan for a number: an integer, double or complex (eg 3i)
        // -------------------------------------------------------------------

        private void getNumber()
        {
            const int MAX_DIGIT_COUNT = 3;  // Max number of digits in exponent

            int single_digit;
            double scale;
            double evalue;
            int exponent_sign;
            int digit_count;

            tokenInteger = 0; tokenDouble = 0.0; tokenScalar = 0.0;
            evalue = 0.0; exponent_sign = 1;

            // Assume first it's an integer
            ftoken = code.tIntToken;

            // Pick up number before any decimal place
            if (fch != '.')
                try
                {
                    do
                    {
                        single_digit = fch - '0';
                        tokenInteger = 10 * tokenInteger + single_digit;
                        tokenScalar = tokenInteger;
                        nextChar();
                    } while (FCharTable[fch] == TCharCode.cDIGIT);
                }
                catch
                {
                    throw new EScannerException("Integer Overflow - constant value too large to read");
                }

            scale = 1;
            if (fch == '.')
            {
                // Then it's a float. Start collecting fractional part
                ftoken = code.tDoubleToken; tokenDouble = tokenInteger;
                nextChar();
                if (FCharTable[fch] != TCharCode.cDIGIT)
                    throw new EScannerException("Syntax error: expecting number after decimal point");

                try
                {
                    while (FCharTable[fch] == TCharCode.cDIGIT)
                    {
                        scale = scale * 0.1;
                        single_digit = fch - '0';
                        tokenDouble = tokenDouble + (single_digit * scale);
                        tokenScalar = tokenDouble;
                        nextChar();
                    }
                }
                catch
                {
                    throw new EScannerException("Floating point overflow - constant value too large to read in");
                }
            }

            // Next check for scientific notation
            if ((fch == 'e') || (fch == 'E'))
            {
                // Then it's a float. Start collecting exponent part
                if (ftoken == code.tIntToken)
                {
                    ftoken = code.tDoubleToken;
                    tokenDouble = tokenInteger;
                    tokenScalar = tokenInteger;
                }
                nextChar();
                if ((fch == '-') || (fch == '+'))
                {
                    if (fch == '-') exponent_sign = -1;
                    nextChar();
                }
                // accumulate exponent, check that first ch is a digit
                if (FCharTable[fch] != TCharCode.cDIGIT)
                    throw new EScannerException("Syntax error: number expected in exponent");

                digit_count = 0;
                try
                {
                    do
                    {
                        digit_count++;
                        single_digit = fch - '0';
                        evalue = 10 * evalue + single_digit;
                        nextChar();
                    } while ((FCharTable[fch] == TCharCode.cDIGIT) && (digit_count <= MAX_DIGIT_COUNT));
                }
                catch
                {
                    throw new EScannerException("Floating point overflow - Constant value too large to read");
                }

                if (digit_count > MAX_DIGIT_COUNT)
                    throw new EScannerException("Syntax error: too many digits in exponent");

                evalue = evalue * exponent_sign;
                if (evalue > 300)
                    throw new EScannerException("Exponent overflow while parsing floating point number");
                evalue = Math.Pow(10.0, evalue);
                tokenDouble = tokenDouble * evalue;
                tokenScalar = tokenDouble;
            }

            // Check for complex number
            if ((fch == 'i') || (fch == 'j'))
            {
                if (ftoken == code.tIntToken)
                    tokenDouble = tokenInteger;
                ftoken = code.tComplexToken;
                nextChar();
            }
        }


        // Returns true if the character ch is a double quote
        private bool IsDoubleQuote(char ch)
        {
            if (FCharTable[ch] == TCharCode.cDOUBLEQUOTE)
                return true;
            else return false;
        }

        // -------------------------------------------------------------------
        // Scan for a string, eg "abc"       
        // -------------------------------------------------------------------       

        private void getString()
        {
            bool OldIgnoreNewLines;
            tokenString = "";
            nextChar();

            ftoken = code.tStringToken;
            while (fch != EOFCHAR)
            {
                // Check for escape characters
                if (fch == '\\')
                {
                    nextChar();
                    switch (fch)
                    {
                        case '\\': tokenString = tokenString + '\\'; break;
                        case 'n': tokenString = tokenString + CR + LF; break;
                        case 'r': tokenString = tokenString + CR; break;
                        case 'f': tokenString = tokenString + LF; break;
                        case 't': tokenString = tokenString + new String(' ', 6); break;
                        default:
                            throw new EScannerException("Syntax error: Unrecognised control code in string");
                    }
                    nextChar();
                }
                else
                {
                    OldIgnoreNewLines = IgnoreNewLines;
                    if (IsDoubleQuote(fch))
                    {
                        // Just in case the double quote is at the end of a line and another string
                        // start immediately in the next line, if we ignore newlines we'll
                        // pick up a double quote rather than the end of a string
                        IgnoreNewLines = false;
                        nextChar();
                        if (IsDoubleQuote(fch))
                        {
                            tokenString = tokenString + fch;
                            nextChar();
                        }
                        else
                        {
                            if (OldIgnoreNewLines)
                            {
                                while (fch == CR)
                                {
                                    nextChar();
                                    while (fch == LF)
                                        nextChar();
                                }
                            }
                            IgnoreNewLines = OldIgnoreNewLines;
                            return;
                        }
                    }
                    else
                    {
                        tokenString = tokenString + fch;
                        nextChar();
                    }
                    IgnoreNewLines = OldIgnoreNewLines;
                }
            }
            if (fch == EOFCHAR)
                throw new EScannerException("Syntax error: String without terminating quotation mark");
        }


        // -------------------------------------------------------------------
        // Scan for special characters
        // -------------------------------------------------------------------

        private void getSpecial()
        {
            char tch;

            switch (fch)
            {
                case CR: ftoken = code.tEolToken;
                    nextChar();
                    break;

                case ';': ftoken = code.tSemiColonToken;
                    nextChar();
                    break;

                case ',': ftoken = code.tCommaToken;
                    nextChar();
                    break;

                case ':': ftoken = code.tColonToken;
                    nextChar();
                    break;

                case '=': nextChar();
                    if (fch == '>')
                    {
                        ftoken = code.tReversibleArrow;
                        nextChar();
                    }
                    else
                        ftoken = code.tEqualsToken;
                    break;

                case '+': ftoken = code.tPlusToken;
                    nextChar();
                    break;

                case '-': nextChar();
                    if (fch == '>')
                    {
                        ftoken = code.tIrreversibleArrow;
                        nextChar();
                    }
                    else
                        ftoken = code.tMinusToken;
                    break;

                case '*': nextChar();
                    ftoken = code.tMultToken;
                    break;

                case '/':  // look ahead at next ch
                    tch = nextChar();
                    if (tch == '/')
                    {
                        ftoken = code.tStartComment;
                        nextChar();
                    }
                    else
                        ftoken = code.tDivToken;
                    break;

                case '(': nextChar();
                    ftoken = code.tLParenToken;
                    break;

                case ')': nextChar();
                    ftoken = code.tRParenToken;
                    break;

                case '[': nextChar();
                    ftoken = code.tLBracToken;
                    break;

                case ']': nextChar();
                    ftoken = code.tRBracToken;
                    break;

                case '{': nextChar();
                    ftoken = code.tLCBracToken;
                    break;

                case '}': nextChar();
                    ftoken = code.tRCBracToken;
                    break;

                case '^': nextChar();
                    ftoken = code.tPowerToken;
                    break;

                case '<': nextChar();
                    if (fch == '=')
                    {
                        ftoken = code.tLessThanOrEqualToken;
                        nextChar();
                    }
                    else
                        ftoken = code.tLessThanToken;
                    break;

                case '>': nextChar();
                    if (fch == '=')
                    {
                        ftoken = code.tMoreThanOrEqualToken;
                        nextChar();
                    }
                    else
                        ftoken = code.tMoreThanToken;
                    break;

                case '!': nextChar();
                    if (fch == '=')
                    {
                        ftoken = code.tNotEqualToken;
                        nextChar();
                    }
                    break;

                case '.': nextChar();
                    ftoken = code.tPointToken;
                    break;

                case '$': nextChar();
                    ftoken = code.tDollarToken;
                    break;

                default:
                    throw new EScannerException("Syntax error: Unknown special token [" + fch + "]");
            }
        }

        public static string[] Operators =
        {
         ";", ",", ":", "=", "+", "-", ">", "*", 
         "/", "(", ")", "[", "]", "{", "}", "^",
         "<", ">", "!", ".", "$"
        };

        // -------------------------------------------------------------------
        // This scanner has a simple queue mechanism that allows one to put
        // tokens back to the scanner via a quaue
        // -------------------------------------------------------------------

        public bool IsQueueEmpty()
        {
            return (tokenQueue.Count == 0);
        }


        // -------------------------------------------------------------------
        // Add the current token to the queue
        // -------------------------------------------------------------------

        public void AddTokenToQueue()
        {
            TToken t = new TToken();
            t.tokenCode = ftoken;
            t.tokenDouble = tokenDouble;
            t.tokenInteger = tokenInteger;
            t.tokenString = tokenString;
            t.tokenValue = tokenScalar;
            tokenQueue.Enqueue(t);
        }

        // -------------------------------------------------------------------
        // Get a token from the queue
        // Check that a token is in the queue first by calling IsQueueEmpty()
        // Used internally
        // -------------------------------------------------------------------

        private void getTokenFromQueue()
        {
            TToken t = (TToken)tokenQueue.Dequeue();

            ftoken = t.tokenCode;
            tokenString = t.tokenString;
            tokenScalar = t.tokenValue;
            tokenInteger = t.tokenInteger;
            tokenDouble = t.tokenDouble;
        }
        // End of Queue routines
        // -------------------------------------------------------------------


        private void nextTokenInternal()
        {
            // check if a token has been pushed back into the token stream, if so use it first
            // I think I should get rid of this code, use queue methods instead
            if (previousToken.tokenCode != code.tEmptyToken)
            {
                ftoken = previousToken.tokenCode;
                tokenString = previousToken.tokenString;
                tokenDouble = previousToken.tokenDouble;
                tokenInteger = previousToken.tokenInteger;
                previousToken.tokenCode = code.tEmptyToken;
                return;
            }

            // Check if there is anything in the token queue, if so get the item
            // from the queue and exit. If not, read as normal from the stream.
            // Checking the queue before reading from the stream can be turned off and on
            // by setting the FromQueue Flag.
            if (FromQueue)
            {
                if (!IsQueueEmpty())
                {
                    getTokenFromQueue();
                    return;
                }
            }

            skipBlanks();
            tokenString = "";

            switch (FCharTable[fch])
            {
                case TCharCode.cLETTER:
                case TCharCode.cUNDERSCORE: getWord(); break;
                case TCharCode.cDIGIT: getNumber(); break;
                case TCharCode.cDOUBLEQUOTE: getString(); break;
                case TCharCode.cETX: ftoken = code.tEndOfStreamToken; break;
                default:
                    getSpecial(); break;
            }
        }

        // -------------------------------------------------------------------
        // Retrieve the next token in the stream, return tEndOfStreamToken
        // if it reaches the end of the stream
        // -------------------------------------------------------------------

        public void nextToken()
        {
            nextTokenInternal();
            while (ftoken == code.tStartComment)
            {
                // Comment ends with an end of line char
                while ((fch != LF) && (fch != EOFCHAR))
                    fch = getCharFromBuffer();
                //while ((fch != CR) && (fch != EOFCHAR)) 
                //      fch = getCharFromBuffer();
                //while ((fch == LF) && (fch !=  EOFCHAR))
                //      fch = getCharFromBuffer();
                while (fch == LF)
                {
                    yylineno++;
                    //while (fch == LF)
                    //      nextChar();  // Dump the linefeed
                    fch = nextChar();
                }
                nextTokenInternal();  // get the real next token
            }
        }


        // -------------------------------------------------------------------
        // Allows one token look ahead
        // Push token back into token stream
        // -------------------------------------------------------------------

        public void UnGetToken()
        {
            previousToken.tokenCode = ftoken;
            previousToken.tokenString = tokenString;
            previousToken.tokenInteger = tokenInteger;
            previousToken.tokenDouble = tokenDouble;
        }

        // -------------------------------------------------------------------
        // Given a token, this function returns the string eqauivalent
        // ------------------------------------------------------------------- 

        public String tokenToString(code code)
        {
            switch (code)
            {
                case code.tIntToken: return "<Integer: " + tokenInteger.ToString() + ">";
                case code.tDoubleToken: return "<Double: " + tokenDouble.ToString() + ">";
                case code.tComplexToken: return "<Complex: " + tokenDouble.ToString() + "i>";
                case code.tStringToken: return "<String: " + tokenString + ">";
                case code.tWordToken: return "<Identifier: " + tokenString + ">";
                case code.tEndOfStreamToken: return "<end of stream>";
                case code.tEolToken: return "<EOLN>";
                case code.tSemiColonToken: return ";";
                case code.tCommaToken: return ",";
                case code.tEqualsToken: return "=";
                case code.tPlusToken: return "+";
                case code.tMinusToken: return "-";
                case code.tMultToken: return "*";
                case code.tDivToken: return "/";
                case code.tPowerToken: return "^";
                case code.tLParenToken: return "(";
                case code.tRParenToken: return ")";
                case code.tLBracToken: return "[";
                case code.tRBracToken: return "]";
                case code.tLCBracToken: return "{";
                case code.tRCBracToken: return "}";
                case code.tDollarToken: return "$";
                case code.tOrToken: return "or";
                case code.tAndToken: return "and";
                case code.tNotToken: return "not";
                case code.tXorToken: return "xor";
                case code.tLessThanToken: return "<";
                case code.tLessThanOrEqualToken: return "<=";
                case code.tMoreThanToken: return ">";
                case code.tMoreThanOrEqualToken: return ">=";
                case code.tNotEqualToken: return "!=";
                case code.tReversibleArrow: return "=>";
                case code.tIrreversibleArrow: return "->";
                case code.tIfToken: return "if";
                case code.tWhileToken: return "while"; ;
                case code.tdefnToken: return "defn";
                case code.tEndToken: return "end";
                case code.tInternalToken: return "Internal";
                case code.tExternalToken: return "External";
                case code.tParameterToken: return "Parameter";
                case code.tTimeStartToken: return "TimeStart";
                case code.tTimeEndToken: return "TimeEnd";
                case code.tNumPointsToken: return "NumPoints";
                case code.tSimulateToken: return "Simulate";

                default:
                    return "<unknown>";
            }
        }
    }  // end of TScanner class


}  // end of uScanner namespace
